Notebook demonstrating how to load data from the OpenScope predictive coding project

This project collected 2-photon calcium imaging data in 3 cortical areas (VISp, VISpm, RSP) across multiple depths in response to several stimulus types, including predictable image sequences with unexpected oddball images, spatial occlusion of varying degrees, natural movies, and control conditions with shuffled sequence and oddball images. In each mouse, a retrograde tracer was injected in either VISp or RSP to enable identification of feedforward and feedback projection neurons between areas. Biometric data including mouse running behavior and pupil diameter were also acquired.

In [1]:
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import seaborn as sns
sns.set_context('notebook', font_scale=1.5, rc={'lines.markeredgewidth': 2})
sns.set_style('white', {'axes.spines.right': False, 'axes.spines.top': False, 'xtick.bottom': False, 'ytick.left': False,})
In [2]:
%load_ext autoreload
%autoreload 2

%matplotlib inline

select an experiment from the manifest

In [3]:
manifest_file = r"C:\Users\marinag\Dropbox\opc_analysis\opc_production_manifest.xlsx"

manifest = pd.read_excel(manifest_file)
In [4]:
# limit to experiments that passed QC
manifest = manifest[manifest['experiment_state']=='passed']
In [5]:
print('total mice:', len(manifest['mouse_id'].unique()))
print('# V1 injections:', len(manifest[manifest['injection_area']=='VISp']['mouse_id'].unique()))
print('# RSP injections:', len(manifest[manifest['injection_area']=='RSP']['mouse_id'].unique()))
print('# V1 imaging sessions:', len(manifest[manifest['imaging_area']=='VISp']['experiment_id'].unique()))
print('# RSP imaging sessions:', len(manifest[manifest['imaging_area']=='RSP']['experiment_id'].unique()))
print('# PM imaging sessions:', len(manifest[manifest['imaging_area']=='VISpm']['experiment_id'].unique()))
total mice: 14
# V1 injections: 7
# RSP injections: 7
# V1 imaging sessions: 16
# RSP imaging sessions: 12
# PM imaging sessions: 22
In [6]:
VISpm_experiments = manifest[(manifest.imaging_area=='VISpm')].experiment_id.values
experiment_id = int(VISpm_experiments[5])
In [7]:
# info for this experiment
manifest[manifest.experiment_id==experiment_id]
Out[7]:
workflow_state experiment_state mouse_id experiment_id session_id genotype imaging_depth imaging_area injection_area acquisition_date name rig operator specimen_id notes project_code failure_tags qc_reviewer
19 uploaded passed 433242 829411383.0 829248349 Slc17a7-IRES2-Cre;Camk2a-tTA;Ai93 385 VISpm RSP 02/26/2019 12:46 20190226_433242_OSPC5 CAM2P.1 sams 802414759 NaN OpenScopePredictiveCoding NaN jenniferl

load data for a single 2-photon imaging session

In [8]:
# data streams for each imaging session are saved as .h5 files in each experiments's folder in this directory
cache_dir = r'C:\Users\marinag\Dropbox\opc_analysis'
In [9]:
# the OpenScopePredictiveCodingDataset class loads the relevant .h5 files and sets the data as attributes
from openscope_predictive_coding.ophys.dataset.openscope_predictive_coding_dataset import OpenScopePredictiveCodingDataset

dataset = OpenScopePredictiveCodingDataset(experiment_id, cache_dir)
In [10]:
from openscope_predictive_coding.ophys.response_analysis.response_analysis import ResponseAnalysis

# set use_events to True if you want responses in events instead of dF/F
analysis = ResponseAnalysis(dataset, use_events=True) 
In [11]:
# plot max intensity projection for this experiment
plt.imshow(dataset.max_projection, cmap='gray')
Out[11]:
<matplotlib.image.AxesImage at 0x1aeedbe1470>
In [12]:
# get cell traces
dataset.dff_traces.head(5)
Out[12]:
dff
cell_specimen_id
829482266 [0.02217028963299241, 0.05111645700839927, 0.0...
829482269 [-0.08213811161298909, 0.02094433551227091, -0...
829482271 [0.055718754591026154, -0.031179706314983166, ...
829482275 [0.07487296291575361, -0.035264481271238435, -...
829482277 [0.07905385120130957, 0.0024523122305217995, -...
In [13]:
# plot a heatmap of all cell traces (this is slow)
fig, ax = plt.subplots(figsize=(20,5))
sns.heatmap(dataset.dff_traces_array, vmax=np.percentile(dataset.dff_traces_array, 99), ax=ax, linecolor=None, linewidth=0)
ax.set_ylabel('cell_index')
ax.set_xlabel('2P frames')
Out[13]:
Text(0.5, 16.5, '2P frames')
In [14]:
def plot_sorted_traces_heatmap(dataset, analysis, ax=None, save=False, use_events=False):
    import openscope_predictive_coding.ophys.response_analysis.response_processing as rp
    import openscope_predictive_coding.ophys.plotting.experiment_summary_figures as esf

    if use_events:
        traces = dataset.events_array.copy()
        traces = rp.filter_events_array(traces, scale=2)
        traces = esf.reorder_traces(traces, analysis)
        vmax = 0.01
        # vmax = np.percentile(traces, 99)
        label = 'event magnitude'
        suffix = '_events'
    else:
        traces = dataset.dff_traces_array
        traces = esf.reorder_traces(traces, analysis)
        vmax = np.percentile(traces, 99)
        label = 'dF/F'
        suffix = ''
    if ax is None:
        figsize = (14, 5)
        fig, ax = plt.subplots(figsize=figsize)

    cax = ax.pcolormesh(traces, cmap='magma', vmin=0, vmax=vmax)
    ax.set_ylabel('cells')

    interval_seconds = 5 * 60
    ophys_frame_rate = int(dataset.metadata.ophys_frame_rate.values[0])
    upper_limit, time_interval, frame_interval = esf.get_upper_limit_and_intervals(traces, dataset.ophys_timestamps,
                                                                               ophys_frame_rate)
    ax.set_xticks(np.arange(0, upper_limit, interval_seconds * ophys_frame_rate))
    ax.set_xticklabels(np.arange(0, upper_limit / ophys_frame_rate, interval_seconds))
    ax.set_xlabel('time (seconds)')

    cb = plt.colorbar(cax, pad=0.015)
    cb.set_label(label, labelpad=3)
    if save:
        esf.save_figure(fig, figsize, dataset.analysis_dir, 'experiment_summary',
                    str(dataset.experiment_id) + 'sorted_traces_heatmap' + suffix)
    return ax
In [ ]:
 
In [15]:
# plot_sorted_traces_heatmap(dataset, analysis, ax=None, save=False, use_events=False)

rastermap

In [16]:
fig, ax = plt.subplots(figsize=(20,5))

sns.heatmap(dataset.dff_traces_array, vmax=np.percentile(dataset.dff_traces_array, 99), ax=ax, linecolor=None, linewidth=0)
Out[16]:
<matplotlib.axes._subplots.AxesSubplot at 0x1aef0395828>
In [17]:
from rastermap import Rastermap
model = Rastermap(n_components=2, n_X=10, nPC=100, init='pca')
# fit does not return anything, it adds attributes to model
# attributes: embedding, u, s, v, isort1
sp = dataset.dff_traces_array.copy()
model.fit(sp)
plt.imshow(sp[model.isort, :])
# fit_transform returns embedding (upsampled cluster identities)
embedding = model.fit_transform(sp)
# transform can be used on new samples with the same number of features as sp
# embed2 = model.transform(sp2)
nmin 72
0.45422863960266113
81.98148345947266
85.19932770729065
85.21233606338501
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
4.70s     0        0.3091      8
4.79s    10        0.7386      48
4.91s    20        0.8274      80
4.99s    30        0.8274      80
5.00s   final      0.8274
30.53s upsampled    0.8274
nmin 72
0.7912018299102783
83.01119565963745
86.64496946334839
86.66798782348633
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.3091      8
0.32s    10        0.7386      48
0.75s    20        0.8274      80
1.19s    30        0.8274      80
1.22s   final      0.8274
28.96s upsampled    0.8274
In [18]:
fig, ax = plt.subplots(figsize=(20,5))
sns.heatmap(sp[model.isort, :], vmax=np.percentile(dataset.dff_traces_array, 99), ax=ax, linecolor=None, linewidth=0)
Out[18]:
<matplotlib.axes._subplots.AxesSubplot at 0x1ae81205400>
In [19]:
sns.heatmap(embedding)
Out[19]:
<matplotlib.axes._subplots.AxesSubplot at 0x1ae812b8f60>

across experiments

In [20]:
cache_dir = r'\\allen\programs\braintv\workgroups\nc-ophys\opc\opc_analysis'
In [21]:
from rastermap import Rastermap
model = Rastermap(n_components=2, n_X=10, nPC=100, init='pca')
# fit does not return anything, it adds attributes to model
# attributes: embedding, u, s, v, isort1

for experiment_id in manifest.experiment_id.unique():
    experiment_id = int(experiment_id)
    try:
        dataset = OpenScopePredictiveCodingDataset(experiment_id, cache_dir)
        sp = dataset.dff_traces_array.copy()
        model.fit(sp)
#         plt.imshow(sp[model.isort, :])
        # fit_transform returns embedding (upsampled cluster identities)
        embedding = model.fit_transform(sp)
        # transform can be used on new samples with the same number of features as sp
        # embed2 = model.transform(sp2)
        fig, ax = plt.subplots(figsize=(20,5))
        ax = sns.heatmap(sp[model.isort, :], vmax=np.percentile(dataset.dff_traces_array, 99), ax=ax, linecolor=None, linewidth=0)
        ax.set_title(dataset.analysis_folder)
    except:
        print('problem for',experiment_id)
nmin 100
2.058182716369629
122.72703003883362
128.77068400382996
128.78919792175293
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.1802      8
0.32s    10        0.5313      48
0.53s    20        0.6735      80
0.90s    30        0.6736      80
0.96s   final      0.6736
28.97s upsampled    0.6736
nmin 100
2.0520873069763184
125.34712672233582
129.9046812057495
129.91466784477234
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.1810      8
0.15s    10        0.5320      48
0.27s    20        0.6725      80
0.47s    30        0.6725      80
0.50s   final      0.6726
32.12s upsampled    0.6726
nmin 60
0.6861510276794434
61.72775888442993
64.53820037841797
64.55026698112488
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.3295      8
0.12s    10        0.7269      48
0.23s    20        0.8171      80
0.35s    30        0.8171      80
0.37s   final      0.8171
25.50s upsampled    0.8171
nmin 60
0.7029852867126465
60.256004333496094
62.62601828575134
62.636085510253906
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.01s     0        0.3295      8
0.10s    10        0.7269      48
0.20s    20        0.8171      80
0.33s    30        0.8171      80
0.37s   final      0.8171
26.64s upsampled    0.8171
nmin 60
0.7319083213806152
57.20197534561157
66.36160612106323
66.37819337844849
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.11s     0        0.9252      8
1.19s    10        0.9677      48
2.38s    20        0.9744      80
3.86s    30        0.9744      80
3.90s   final      0.9744
62.83s upsampled    0.9744
nmin 60
1.5051870346069336
109.99548077583313
114.76340007781982
114.79139399528503
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.08s     0        0.9250      8
1.09s    10        0.9677      48
1.88s    20        0.9745      80
2.41s    30        0.9745      80
2.43s   final      0.9745
63.10s upsampled    0.9745
nmin 60
1.385408639907837
103.9402117729187
108.8807647228241
108.90728545188904
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.26s     0        0.2315      8
0.88s    10        0.6710      48
1.49s    20        0.7926      80
2.72s    30        0.7926      80
2.98s   final      0.7926
63.91s upsampled    0.7926
nmin 60
1.8647828102111816
77.41511154174805
82.58184385299683
82.60235905647278
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.04s     0        0.2307      8
0.79s    10        0.6630      48
1.19s    20        0.7788      80
1.99s    30        0.7788      80
2.02s   final      0.7788
28.16s upsampled    0.7788
nmin 60
1.2071287631988525
60.2740797996521
63.30381679534912
63.33590579032898
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.2309      8
0.17s    10        0.6656      48
0.55s    20        0.7839      80
0.85s    30        0.7839      80
0.88s   final      0.7839
27.03s upsampled    0.7839
nmin 60
0.9897041320800781
61.22053003311157
64.2130618095398
64.22457027435303
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.02s     0        0.2323      8
0.16s    10        0.6642      48
0.30s    20        0.7899      80
0.48s    30        0.7900      80
0.49s   final      0.7900
24.98s upsampled    0.7900
nmin 60
0.8019964694976807
60.88966155052185
64.35638999938965
64.36896228790283
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.01s     0        0.3308      8
0.15s    10        0.7074      48
0.44s    20        0.8048      80
0.77s    30        0.8048      80
0.81s   final      0.8048
24.58s upsampled    0.8048
nmin 60
0.94222092628479
61.220377683639526
64.25943183898926
64.27299046516418
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.01s     0        0.3321      8
0.14s    10        0.7055      48
0.38s    20        0.8044      80
0.75s    30        0.8044      80
0.77s   final      0.8044
26.34s upsampled    0.8044
nmin 60
2.951204538345337
60.511088848114014
62.84318161010742
62.854188680648804
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.01s     0        0.3158      8
0.06s    10        0.7410      48
0.09s    20        0.8295      80
0.14s    30        0.8295      80
0.16s   final      0.8295
8.50s upsampled    0.8295
nmin 60
1.7013978958129883
41.62244772911072
43.21177792549133
43.21777701377869
(80, 100)
(32,)
1.0
time; iteration;  explained PC variance
0.00s     0        0.3175      8
0.06s    10        0.7462      48
0.12s    20        0.8305      80
0.16s    30        0.8305      80
0.17s   final      0.8305
8.37s upsampled    0.8305
problem for 818894752
problem for 826576489
problem for 827232898
problem for 828956958
problem for 829411383
problem for 848005700
problem for 848690810
problem for 848006710
problem for 848691390
problem for 830688102
problem for 832601977
problem for 832617299
problem for 833599179
problem for 830075254
problem for 830688059
problem for 831312165
problem for 832107135
problem for 833614835
problem for 834260382
problem for 838330377
problem for 835642229
problem for 835654507
problem for 836246273
problem for 836891984
problem for 833626456
problem for 836248932
problem for 837630919
problem for 837287590
problem for 827235482
problem for 828959377
problem for 829417358
problem for 831314921
problem for 833612445
problem for 835660148
problem for 836253258
problem for 836906598
problem for 834244626
problem for 836250018
problem for 837285285
problem for 833611925
problem for 834251985
problem for 836890936
problem for 837283374
In [22]:
model.isort
Out[22]:
array([167,  42, 122,  57, 133, 126,  67,  86, 228, 221, 159, 151,  56,
        98,  35,  84, 163, 217,   7, 182,  55, 200, 186,  21, 138,  99,
       113,   3, 175,  16, 135,  68, 209, 170, 235, 207, 137, 173, 194,
        73,  41, 146,  60,  65,  24, 198,  11,  59, 161,  14, 185, 184,
        79,  30, 121, 205,  22,  91,  88, 201,  54, 213,  29,  82, 100,
        93, 101, 103,  81,  37, 211,  49,  52,  75, 231, 236, 225, 199,
       144,   4, 183,  39, 164, 196,  53, 233,  50,  13, 106,  32,  80,
         9, 134, 156,  47,  83, 147, 166, 104, 212,  58,  28, 188, 130,
       218, 124, 114,   2, 193, 203, 187, 190,  61, 107,  62, 150, 120,
        43,  85, 162, 210, 168,  33, 128, 154,   0, 145,  15, 179, 176,
        36, 152, 132,  76,  31, 148, 234, 110, 102, 108,  19, 216,  77,
        71, 129, 153,  92,  66,   8, 118, 208, 223,  38,  40,  20, 174,
       197, 127, 171, 116, 155,  12,  89, 215, 117,  23,  90, 141,  45,
       214, 109,   5, 123,  97,   1, 232, 177, 149,  44, 112, 160,  26,
       230, 140,  96, 142, 229, 189, 105, 158,  94, 172, 119,  48, 139,
        74,  64, 169,  78, 180, 136,  25, 227,  34,  18,  70, 224, 178,
       191, 202,  63, 131, 143,  95,  17, 111,  10, 115,   6, 192, 226,
       157, 181,  69,  72, 219, 222, 204,  51, 237,  27,  87, 238,  46,
       165, 206, 125, 195, 220], dtype=int64)
In [ ]:
dataset.metadata
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
# minmax standardized
In [ ]:
methods = ['single', 'average', 'weighted', 'median', 'ward']
metrics = ['euclidean', 'correlation', 'mahalanobis']
In [ ]:
sns.clustermap(dataset.dff_traces_array, method='average', metric='correlation', z_score=None, standard_scale=None)
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
for method in methods: 
    for metric in metrics:
        try:
            fig, ax = plt.subplots()
            ax = sns.clustermap(dataset.dff_traces_array, method='average', metric='euclidean', z_score=None, standard_scale=0, ax=ax)
            ax.set_title(method+', '+metric+', standard scale')
        except:
            pass
In [ ]:
sns.clustermap(dataset.dff_traces_array, method='average', metric='euclidean', z_score=None, standard_scale=0,)
In [ ]:
sns.clustermap(dataset.dff_traces_array, method='average', metric='euclidean', z_score=0, standard_scale=None)
In [ ]:
 
In [ ]:
 
In [14]:
# plot one cell's dF/F trace with x-axis in time in seconds 
cell_specimen_id = dataset.cell_specimen_ids[0]
dff_trace = dataset.dff_traces.loc[cell_specimen_id].values[0]
timestamps = dataset.timestamps_ophys
fig, ax = plt.subplots(figsize=(20,3))
ax.plot(timestamps, dff_trace)
ax.set_ylabel('dF/F')
ax.set_xlabel('time (sec)')
Out[14]:
Text(0.5, 0, 'time (sec)')
In [15]:
# plot deconvolved calcium events for the same cell
events_trace = dataset.events.loc[cell_specimen_id].values[0]
timestamps = dataset.timestamps_ophys
fig, ax = plt.subplots(figsize=(20,3))
ax.plot(timestamps, events_trace)
ax.set_ylabel('dF/F')
ax.set_xlabel('time (sec)')
getting L0 events
Out[15]:
Text(0.5, 0, 'time (sec)')
In [16]:
# plot mouse running speed with x-axis in seconds
run_speed = dataset.running_speed.speed.values
timestamps = dataset.running_speed.time.values
fig, ax = plt.subplots(figsize=(20,3))
ax.plot(timestamps, run_speed)
ax.set_ylabel('running speed (cm/s)')
ax.set_xlabel('time (sec)')
Out[16]:
Text(0.5, 0, 'time (sec)')

looks like this mouse didnt run

In [17]:
# what are the different stimulus types? 
dataset.stimulus_table.session_block_name.unique()
Out[17]:
array(['randomized_control_pre', 'oddball', 'transition_control',
       'occlusion', 'natural_movie_one', 'randomized_control_post'],
      dtype=object)
In [18]:
# plot a cell trace with duration of stimulus blocks indicated
dff_trace = dataset.dff_traces.loc[cell_specimen_id].values[0]
timestamps = dataset.timestamps_ophys
fig, ax = plt.subplots(figsize=(20,3))
ax.plot(timestamps, dff_trace)
ax.set_ylabel('dF/F')
ax.set_xlabel('time (sec)')

block_df = dataset.stimulus_block_table.copy()
colors = sns.color_palette('deep')
for i, block_name in enumerate(block_df.block_name.values):
    start_time = block_df[block_df.block_name==block_name].start_time.values[0]
    end_time = block_df[block_df.block_name==block_name].end_time.values[0]
    ax.axvspan(start_time, end_time, facecolor=colors[i], edgecolor='none', alpha=0.3, linewidth=0, zorder=1, label=block_name)
ax.legend(bbox_to_anchor=(1,1))
Out[18]:
<matplotlib.legend.Legend at 0x118b3b2d780>
In [19]:
# stimulus table has times of all stimulus presentations for each of the stimulus types
dataset.stimulus_table.head(3)
Out[19]:
index sweep start_frame end_frame_inclusive start_time end_time session_block_name image_id repeat fraction_occlusion duration session_type stimulus_key data_file_index data_file_name frame_list mean_running_speed
stimulus_presentations_id
0 0 0 3600 3614 97.21053 97.44403 randomized_control_pre 26.0 NaN NaN 0.25 A ophys_pilot_randomized_control_A 0 //allen/aibs/technology/nicholasc/openscope/op... [3600, 3601, 3602, 3603, 3604, 3605, 3606, 360... 0.071720
1 1 1 3615 3629 97.46070 97.69429 randomized_control_pre 68.0 NaN NaN 0.25 A ophys_pilot_randomized_control_A 1 //allen/aibs/technology/nicholasc/openscope/op... [3615, 3616, 3617, 3618, 3619, 3620, 3621, 362... -0.282074
2 2 2 3630 3644 97.71094 97.94444 randomized_control_pre 6.0 NaN NaN 0.25 A ophys_pilot_randomized_control_A 2 //allen/aibs/technology/nicholasc/openscope/op... [3630, 3631, 3632, 3633, 3634, 3635, 3636, 363... 0.009330

get cell responses for each stimulus presentation

the ResponseAnalysis class does the work of temporal alignment between 2P times and stimulus times and creates dataframes with the response of every cell for every stimulus in a given stimulus block type. If these dataframes have already been generated, the get_response_df() method loads the saved dataframe.

In [20]:
from openscope_predictive_coding.ophys.response_analysis.response_analysis import ResponseAnalysis
In [21]:
# set use_events to True if you want responses in events instead of dF/F
analysis = ResponseAnalysis(dataset, use_events=False) 
In [22]:
# what are the stimulus types again? 
dataset.stimulus_block_table
Out[22]:
block_name start_frame end_frame start_time end_time
0 natural_movie_one 201900 219898 3404.84214 3705.04772
1 occlusion 162330 198270 2744.81699 3344.29405
2 oddball 13500 133485 262.34167 2263.68416
3 randomized_control_post 223500 229785 3765.12887 3869.96230
4 randomized_control_pre 3600 9885 97.21053 202.04375
5 transition_control 137100 158685 2323.98217 2684.01859
In [23]:
# provide the block name to this method to get the stimulus triggered responses for that stimulus type
odf = analysis.get_response_df('oddball')
loading response dataframe for oddball
C:\Users\marinag\AppData\Roaming\Python\Python37\site-packages\pandas\core\indexing.py:376: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self.obj[key] = _infer_fill_value(value)
C:\Users\marinag\AppData\Roaming\Python\Python37\site-packages\pandas\core\indexing.py:494: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self.obj[item] = s
In [24]:
odf.head(3)
Out[24]:
stimulus_presentations_id cell_specimen_id trace trace_timestamps mean_response baseline_response p_value index sweep start_frame ... duration session_type stimulus_key data_file_index data_file_name frame_list mean_running_speed oddball sequence_start violation_sequence
0 420 829482266 [0.020222490744390212, -0.055988010690977655, ... [-1.990613291893292, -1.9574364036950704, -1.9... 0.017578 -0.009745 0.4170 0 420 13500 ... 0.25 A (68, 78, 13, 26) 0 //allen/aibs/technology/nicholasc/openscope/68... [13500, 13501, 13502, 13503, 13504, 13505, 135... -0.048806 False True False
1 420 829482269 [-0.06192142547664995, 0.04304486341351953, -0... [-1.990613291893292, -1.9574364036950704, -1.9... -0.015661 -0.005818 0.8012 0 420 13500 ... 0.25 A (68, 78, 13, 26) 0 //allen/aibs/technology/nicholasc/openscope/68... [13500, 13501, 13502, 13503, 13504, 13505, 135... -0.048806 False True False
2 420 829482271 [0.02342878294005412, 0.046452763256736654, 0.... [-1.990613291893292, -1.9574364036950704, -1.9... 0.018385 0.016577 0.3660 0 420 13500 ... 0.25 A (68, 78, 13, 26) 0 //allen/aibs/technology/nicholasc/openscope/68... [13500, 13501, 13502, 13503, 13504, 13505, 135... -0.048806 False True False

3 rows × 27 columns

In [25]:
odf.keys()
Out[25]:
Index(['stimulus_presentations_id', 'cell_specimen_id', 'trace',
       'trace_timestamps', 'mean_response', 'baseline_response', 'p_value',
       'index', 'sweep', 'start_frame', 'end_frame_inclusive', 'start_time',
       'end_time', 'session_block_name', 'image_id', 'repeat',
       'fraction_occlusion', 'duration', 'session_type', 'stimulus_key',
       'data_file_index', 'data_file_name', 'frame_list', 'mean_running_speed',
       'oddball', 'sequence_start', 'violation_sequence'],
      dtype='object')
In [26]:
# plot single trial response for one cell
trace = odf[(odf.cell_specimen_id==cell_specimen_id)].trace.values[50]
plt.plot(trace)
plt.xlabel('2P frames')
plt.ylabel('dF/F')
Out[26]:
Text(0, 0.5, 'dF/F')
In [27]:
# plot trial averaged response for all oddball images for one cell
mean_trace = odf[(odf.cell_specimen_id==cell_specimen_id)&(odf.oddball==True)].trace.mean()
times = odf[(odf.cell_specimen_id==cell_specimen_id)&(odf.oddball==True)].trace_timestamps.values[0]

fig, ax = plt.subplots()
ax.plot(times, mean_trace)
ax.set_xlabel('time after stimulus onset (sec)')
ax.set_ylabel('dF/F')
Out[27]:
Text(0, 0.5, 'dF/F')
In [28]:
# plot the populatin response vector for one trial - mean response of all cells to that stimulus
# stimulus_presentations_id is the index of each individual stimulus presentation across the entire session
oddball_trials = odf[odf.oddball==True].stimulus_presentations_id.unique()
mean_responses = odf[(odf.stimulus_presentations_id==oddball_trials[9])].mean_response.values
plt.plot(mean_responses, '.')
plt.xlabel('cells')
plt.ylabel('mean response')
plt.title('population response for one oddball trial')
Out[28]:
Text(0.5, 1.0, 'population response for one oddball trial')
In [29]:
# how many oddball images are there anyway? 
print(len(analysis.oddball_images), 'oddball images')
print('oddball image IDs are', analysis.oddball_images)
10 oddball images
oddball image IDs are [6.0, 17.0, 22.0, 51.0, 71.0, 89.0, 103.0, 110.0, 111.0, 112.0]
In [30]:
# what are the sequence images? 
print('sequence image IDs are', analysis.sequence_images)
sequence image IDs are [68.0, 78.0, 13.0, 26.0]
In [31]:
# plot one of the images 
stimulus_template = dataset.get_stimulus_template()
image_id = analysis.sequence_images[1]
plt.imshow(stimulus_template[image_id])
Out[31]:
<matplotlib.image.AxesImage at 0x118b39a54e0>

get trial averaged responses for a set of conditions

In [32]:
# start with dataframe with every cell's response to every stimulus presentation
odf = analysis.get_response_df('oddball')
loading response dataframe for oddball
In [33]:
# utilities has some useful functions for trial averaging, determining significance of response, etc. 
import openscope_predictive_coding.ophys.response_analysis.utilities as ut
In [34]:
# the get_mean_df() function gets trial averaged responses and other metrics for a set of conditions 
conditions=['cell_specimen_id', 'oddball', 'image_id'] # columns in response dataframe over which to groupby
oddball_mean_df = ut.get_mean_df(odf, conditions=conditions)
In [35]:
oddball_mean_df.head()
Out[35]:
index cell_specimen_id oddball image_id mean_response sem_response mean_trace sem_trace pref_stim fraction_significant_trials fraction_responsive_trials
0 0 829482266 False 13.0 0.019421 0.002238 [0.018599499249567462, 0.017350379152847723, 0... [0.00195665688100999, 0.0022853478889058314, 0... False 0.017000 0.042000
1 1 829482266 False 26.0 0.014624 0.002069 [0.022781207802966785, 0.01966453978135488, 0.... [0.002888825835157495, 0.0028417273214166486, ... False 0.012632 0.035789
2 2 829482266 False 68.0 0.012886 0.001497 [0.012312569810942151, 0.012522726916159937, 0... [0.0021874886622547393, 0.0020502090220063844,... False 0.008000 0.033000
3 3 829482266 False 78.0 0.016918 0.001651 [0.010273554304274077, 0.011579506693316923, 0... [0.001659922848022397, 0.0015686741350607934, ... False 0.012500 0.041000
4 4 829482266 True 6.0 0.026478 0.006619 [0.02163839779939416, -0.006000974697554415, -... [0.00963872610068699, 0.01292199048345216, 0.0... False 0.000000 0.000000
In [36]:
# plot the population average response for oddball and sequence images
last_in_sequence = analysis.sequence_images[-1] 
plt.plot(oddball_mean_df[oddball_mean_df.oddball==True].mean_trace.mean(), label='oddball')
plt.plot(oddball_mean_df[oddball_mean_df.image_id==last_in_sequence].mean_trace.mean(), label='sequence')
plt.title('population average response')
plt.xlabel('2P frames')
plt.ylabel('dF/F')
plt.legend()
Out[36]:
<matplotlib.legend.Legend at 0x11aa39369e8>
In [37]:
# plot mean response across oddball images for some cell
cell_data = oddball_mean_df[oddball_mean_df.cell_specimen_id==cell_specimen_id]
for image_id in analysis.oddball_images: 
    trace = cell_data[cell_data.image_id==image_id].mean_trace.values[0]
    plt.plot(trace, label=int(image_id))
plt.xlabel('2P frames')
plt.ylabel('dF/F')
plt.title('cell '+str(cell_specimen_id)+'\nmean oddball image responses')
plt.legend(bbox_to_anchor=(1,1), fontsize='medium', title='image_id')
Out[37]:
<matplotlib.legend.Legend at 0x118d96ee0b8>
In [ ]:
 
In [ ]: